{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定位重复 &\n",
    "with open('./Label/data.txt', 'r', encoding=\"utf-8\") as f:\n",
    "    for line in f:\n",
    "        if line.count('&') >= 2:\n",
    "            print(line.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "# 数据清洗\n",
    "with open('merge.txt', 'r', encoding='utf-8') as f:\n",
    "    content = f.read()\n",
    "sentences = re.findall(r'[^。？！；\\n]+[。？！；\\n]', content)\n",
    "\n",
    "# 统计每个句子在原字符串中出现的次数\n",
    "sentence_count = {}\n",
    "for sentence in sentences:\n",
    "    count = content.count(sentence)\n",
    "    sentence_count[sentence] = count\n",
    "    \n",
    "for sentence, count in sentence_count.items():\n",
    "    if count > 5:\n",
    "        print(f'{sentence.strip()}  --- {count}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModel\n",
    "\n",
    "# 在线加载模型\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"hfl/chinese-roberta-wwm-ext\")\n",
    "model = AutoModel.from_pretrained(\"hfl/chinese-roberta-wwm-ext\")"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
